import os
from concurrent.futures.thread import ThreadPoolExecutor
import time
import subprocess
from termcolor import cprint
from pathlib import Path
from hashlib import md5
from pymongo import MongoClient
import concurrent

client = MongoClient()

J = os.path.join
E = os.path.exists
B = os.path.basename

TMP_TEST_DIR = "/tmp/testpy"
if  not E(TMP_TEST_DIR):
    os.mkdir(TMP_TEST_DIR)

SCANER_PDF = "pdftotext"
SCANER_DOC = "docx2txt"

def get_meta(path):
    st = os.stat(path)
    return {
        "value":st.st_mtime,
        "name": os.path.abspath(path),
        "info_type": "record"
    }


def get_old(client):
    all_rec = {}
    for doc in client.sbas.meta.find({
        "info_type" : "record"
    }):
        all_rec[doc['name']] = doc['value']
    return all_rec

def upsert(client, *new_records):
    for rec in new_records:
        client.sbas.meta.update_one({"name":rec['name']}, {"$set": rec}, upsert=True)
    

def do_pdf(path):
    try:
        p = Path(path)
        to_save = J(TMP_TEST_DIR, os.path.basename(path)) +  ".nlp"
        
        res = subprocess.check_output([SCANER_PDF,str(p), to_save])
        # print(res)
        # os.popen(f"{SCANER} {str(p)} {to_save}").read()
        # print(B(path), "->", B(to_save), end="\r")
    except subprocess.CalledProcessError as e:
        pass
        # print("error pdf:",path)
        # print(res,e)
    except Exception as e:
        # print(path, e)
        pass
    finally:
        print("---------- ", path)

def do_docx(path):
    try:
        p = Path(path)
        to_save = J(TMP_TEST_DIR, os.path.basename(path)) + ".nlp"
        res = subprocess.check_output([SCANER_DOC,str(p), " >> ", to_save])
        
    except subprocess.CalledProcessError as e:
        cprint(path, "red")
    except Exception as e:
        cprint(path, "red")


def do_some(path):
    if path.endswith(".pdf"):
        do_pdf(path)
    elif path.endswith(".docx"):
        do_docx(path)


def main():

    old_records = get_old(client)
    new_records = []
    
    c = 0
    start_t = time.time()
    last_time = time.time()
    print("start time : ", time.asctime())
    
    
    with ThreadPoolExecutor(5) as exe:
        fus = {}
        for root, _, files in os.walk(J(os.getenv("HOME"),"Documents/workspace")):
            if os.path.dirname(root).startswith("."):continue
            for file in files:
                path = J(root, file)
                c += 1
                if file.endswith(".pdf") or file.endswith(".docx"):
                    new_data = get_meta(path)
                    if path not in old_records or old_records[path] != new_data['value']:
                        fus[exe.submit(do_some, path)] = path
                        new_records.append(new_data)

                    if time.time() - last_time > 5:
                        print("time:",time.asctime(), "spawn:", c, end='\r')
                        last_time = time.time()
        
        for f in concurrent.futures.as_completed(fus):
            p = fus[f]
            f.result()
    print("end :", time.asctime())
    upsert(client, *new_records)
    
    print("ued :", time.time() - start_t)

if __name__ == "__main__":
    main()